#!/usr/bin/python
"""
cre_lex.py

Lexer for Consistent Regular Expressions.

States:
  start
  character classes
  repetitions
  repetitions spec

Coloring:
  - Literals (char and string are the same?)
  - Repetition operators
  - other operators

TODO:

- Self host?  Generate cre_lex_bootstrap or something.  With a bunch of vars.
  - Generate it with a shell script.
"""

from pprint import pprint

import lexer

SimpleToken = lexer.SimpleToken
SimpleTokenType = lexer.SimpleTokenType
ReEdge = lexer.ReEdge


class Error(Exception):
  pass


# Signature:
#
# Args:
#   regex match object
#   context: lexer state.  You can change states here?  Or push, pop, etc.
# Returns:
#   (token type, token value)
#
# ID is automatically assigned.  position is automatically put in the table.

def Digit(match, context):
  """Convert to integer."""
  return ('DIGIT', int(match.group(0)))


# Char literals can be used outside char classes, or inside char classes.  And
# as part of ranges.

def HexDigit(match, context):
  """Convert to byte literal."""
  d = int(match.group(1), base=16)
  # use unichr, \0x00 is the same as \u0000 in Python.  In Annex, we also send
  # the re engine a unicode string?
  return ('CHAR_LITERAL', unichr(d))


def UnicodeChar(match, context):
  """Convert to unicode char literal."""
  d = int(match.group(1), base=16)
  return ('CHAR_LITERAL', unichr(d))


# When the function is None, it means IGNORE the token.

# state -> (expression, action, new state) list

# These are valid both inside and outside character classes.  AND they can be
# negated both inside and outside.

PERL = ReEdge(
    r'(wordchar|digit|whitespace)\b', SimpleToken('PERL'), kind='NamedClass')

UNICODE_CLASS = ReEdge(
    r'::(\w+)\b', SimpleToken('UNICODE', group=1), kind='NamedClass')

POSIX = ReEdge(
    r':([a-z]+)\b', SimpleToken('POSIX', group=1), kind='NamedClass')

UNICODE_CHAR = ReEdge(
    r'&([0-9a-fA-F]{4})\b', UnicodeChar, kind='CharLiteral')

HEX_CHAR = ReEdge(
    r'0x([0-9a-fA-F]{2})\b', HexDigit, kind='CharLiteral')

# space: inside char class, outside could be ' '
# tab, newline, cr: equivalent of \t \n \r
# '#' starts a comment inside a character class
# '!' negation
# '-' range
# ']' close char class
#     lbracket just for symmetry
# squote: syntactic sugar for '"'
# dquote: syntactic sugar for "'"

SINGLE = ReEdge(
    r'&(space|tab|newline|cr|hash|bang|hyphen|rbracket|lbracket|squote|dquote)\b',
    SimpleToken('SINGLE', group=1), kind='CharLiteral')

# This should stop tokenization
# Inside a char class, you should use & with a space after it to
# signify an ampersand.  Outside a char class, this wouldn't be valid anyway, so
# give a more specific error.

CHAR_ERROR = ReEdge(
    r'&\w+', lexer.ErrorToken('Invalid notation for character: %s'))

# Prevent against typos in hex codes.  We don't want 0xa or 0xag to be 3
# or 4 characters inside a char class.
HEX_ERROR = ReEdge(
    r'0x\w+', lexer.ErrorToken('Invalid notation for character: %s'))

# Ignore comments until the end of the line.
# Have to escape # because of re.VERBOSE.
# Using \n as line ending will work for \r\n -- don't care about plain
# \r.
COMMENT = ReEdge(r'[#] .* \n', None, kind='Comment')  # '#' any* \n


MACHINE = {
    'start': [
        # space is consumed then ignored.
        (r'\s+',),  # space+

        COMMENT,

        # Change state to character class.
        (ReEdge(r'\[', SimpleTokenType, kind='Operator'), 'class'),

        # keywords:
        ReEdge(
            r'(any|chars|either|or|as|flags|REF|IF|THEN|ELSE)\b',
            SimpleTokenType, kind='Keyword'),

        SINGLE,  # starts with &

        PERL,  # keyword-style
        UNICODE_CLASS,  # start with ::
        POSIX,  # start with :

        # hex digit stands for a BYTE.  I don't think we need octal.  That's
        # too old school.
        HEX_CHAR,  # start with 0x

        # Unicode code point.
        UNICODE_CHAR,  # start with &

        (r'%ASSERT(LEFT)?', SimpleToken('ASSERT', group=1)),

        # Right now this is only used inside REF(1) in the main state.
        ReEdge(r'\d+', Digit, kind='Digit'),

        ReEdge(r'[a-zA-Z_][a-zA-Z0-9_]*', SimpleToken('IDENTIFIER'),
               kind='Identifier'),

        # List them out here?
        # begin / end / boundary are in Python
        # begin-word \< and end-word \> are in egrep, but NOT python.
        ReEdge(r'%([a-z\-]+)', SimpleToken('ZERO', group=1), kind='Zero'),

        # ^3, ^(3..4) requires another state.
        (ReEdge(r'\^', SimpleTokenType, kind='RepeatOperator'), 'repeat'),

        # shortcut for nongreedy operations.
        ReEdge(r'\+\+ | \*\* | \?\?', SimpleTokenType, kind='RepeatOperator'),

        # Single character operators.
        ReEdge(r'[=!(){}]', SimpleTokenType, kind='Operator'),

        # Single character repeat operators.  This is separate for syntax
        # highlighting (not lexing).
        ReEdge(r'[+*?]', SimpleTokenType, kind='RepeatOperator'),

        # double quote up to first backslash
        ReEdge(
            r'  " ([^"]*) "  ',
            SimpleToken('LITERAL', group=1),
            kind='Literal'),
        # single quote up to first backslash
        ReEdge(
            r"  ' ([^']*) '  ", 
            SimpleToken('LITERAL', group=1),
            kind='Literal'),

        CHAR_ERROR,
        HEX_ERROR,
        ],

    'class': [
        # whitespace is ignored inside character classes
        (r'\s+', None),

        COMMENT,

        # negation or hyphen are operators.
        ReEdge(r'[!\-]', SimpleTokenType, kind='Operator'),

        # 'any' not allowed since . isn't special inside classes.

        SINGLE,  # starts with &

        PERL,  # keyword-style
        UNICODE_CLASS,  # start with ::
        POSIX,  # start with :
        HEX_CHAR,  # start with 0x
        UNICODE_CHAR,  # start with &

        # These come BEFORE \S+
        CHAR_ERROR,
        HEX_ERROR,

        # character clump
        (r'[^\s\]\-]{2,}', SimpleToken('CLUMP')),
        # a single character, valid in a range or by itself
        (r'[^\s\]]', SimpleToken('CHAR')),

        (ReEdge(r'\]', SimpleTokenType, kind='Operator'), 'start'),
        ],

    # Choose between foo^3 or foo^( arbitrary spec )
    'repeat': [
        (r'\s+',),  # ignore space
        # Used for digit^3 now.  Back to start state.
        (ReEdge(r'\d+', Digit, kind='Digit'), 'start'),
        # open paren -- now it is an arbitrary repeat-spec.
        (r'\(', SimpleTokenType, 'repeat-spec'),
        ],

    # this whole thing should be the same color?  no tokens?  maybe the .. and ?
    # operators?
    'repeat-spec': [
        (r'\s+',),  # ignore space

        # Used for digit^(3..4)
        ReEdge(r'\d+', Digit, kind='Digit'),
        # repetitions
        (r'[*+?]', SimpleTokenType),
        # range
        (r'\.\.', SimpleTokenType),
        # nongreedy or possessive
        (r'[NP]', SimpleToken('MODIFIER')),
        # ( enters repeat-spec state, ) exits it.
        (r'\)', SimpleTokenType, 'start'),
        ]
    }


_lexer = None

def GetLexer():
  global _lexer
  if _lexer:
    return _lexer

  # For bootstrapping
  _lexer = lexer.Lexer(MACHINE, use_re=True)
  return _lexer
